import requests
import re
import time
import pymysql
import random
import gc

header = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,la;q=0.7',
    'Cache-Control': 'max-age=0',
    'Connection': 'keep-alive',
    'Cookie': 'bid=GUzNyyNoJOQ; douban-fav-remind=1; gr_user_id=4bb688d7-744c-412d-b9f3-223e6122aa39; gr_session_id_22c937bbd8ebd703f2d8e9445f7dfd03=0bac908b-b503-416e-ad77-853ebcc89a5e; gr_cs1_0bac908b-b503-416e-ad77-853ebcc89a5e=user_id%3A0; _pk_ref.100001.3ac3=%5B%22%22%2C%22%22%2C1551253192%2C%22https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3Dt720BeJUOC-jTdEgctOeORMPK07ow2M4Br60duF7x2KxJfjfJaf3CYbQV80hwy_3%26ie%3Dutf-8%26f%3D8%26tn%3Dbaidu%26wd%3D%25E8%25B1%2586%25E7%2593%25A3%25E4%25B9%25A6%25E7%25B1%258D%26oq%3Djd%26rqlang%3Dcn%26inputT%3D9766%22%5D; _pk_ses.100001.3ac3=*; ap_v=0,6.0; gr_session_id_22c937bbd8ebd703f2d8e9445f7dfd03_0bac908b-b503-416e-ad77-853ebcc89a5e=true; __utma=30149280.1733359555.1550478802.1550903301.1551253193.3; __utmc=30149280; __utmz=30149280.1551253193.3.3.utmcsr=baidu|utmccn=(organic)|utmcmd=organic|utmctr=%E8%B1%86%E7%93%A3%E4%B9%A6%E7%B1%8D; __utmt_douban=1; __utma=81379588.1437008468.1551253193.1551253193.1551253193.1; __utmc=81379588; __utmz=81379588.1551253193.1.1.utmcsr=baidu|utmccn=(organic)|utmcmd=organic|utmctr=%E8%B1%86%E7%93%A3%E4%B9%A6%E7%B1%8D; __utmt=1; viewed="25862578"; _vwo_uuid_v2=D52887EC4BC6FB6E807FF77B1654FC169|c78e24cc45b9156ea424e7d50ab8e1e6; __utmb=30149280.12.10.1551253193; __utmb=81379588.12.10.1551253193; _pk_id.100001.3ac3=79c6f3e00bcf10ed.1551253192.1.1551253391.1551253192.',
    'Host': 'book.douban.com',
    'Referer': 'https://book.douban.com/tag/?view=cloud',
    'Upgrade-Insecure-Requests': '1',
}
user_agent = [
    "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
    "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
    "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0",
    "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729; InfoPath.3; rv:11.0) like Gecko",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)",
    "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
    "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
    "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11",
    "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
    "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
    "Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
    "Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn; MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10",
    "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
    "Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 Mobile Safari/534.1+",
    "Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.0; U; en-US) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/233.70 Safari/534.6 TouchPad/1.0",
    "Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18124",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)",
    "UCWEB7.0.2.37/28/999",
    "NOKIA5700/ UCWEB7.0.2.37/28/999",
    "Openwave/ UCWEB7.0.2.37/28/999",
    "Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/999",
    "Mozilla/6.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/8.0 Mobile/10A5376e Safari/8536.25",
]

# url = 'https://book.douban.com/subject/1770782/'
# header['User-Agent'] = random.choice(user_agent)
# page = requests.get(url, headers=header)
# page.encoding = 'utf-8'
# print(page.text)
# print(len(re.findall('内容简介', page.text)))
# print(len(re.findall('safasdfsdfsdafsfsdfdsfdsfdsf', page.text)))

url = 'https://book.douban.com/subject/3297466/'

regx_get_book_info = [r'<span property="v:itemreviewed">(.*?)</span>',
                      r'<div.id="info".class="">([\s\S]*?)</div>',
                      r'<a.class="nbg"[\s\S]*?href="(.*?)".title=".*?">',
                      r'<strong class=".*?" property="v:average">(.*?)</strong>',
                      r'(<div class="intro">[\s\S][^void]*?</div>)',
                      r'(<div class="indent" id="dir_.*?_full" style="display:none">[\s\S]*?</div>)',
                      r'tag/(.*?)\?',
                      r'>(.*?)<',
                      r'内容简介']

sub_regx = [r'\n|&nbsp|;| |:']


def get_info_by_list(info, flag):
    tmp = []
    for i in info:
        if len(i) != 0:
            tmp.append(i)

    tmp_arr = []
    for i in range(0, len(tmp), 2):
        if tmp[i] in '作者':
            flag['author'] = tmp[i + 1]
        elif tmp[i] in '出版社':
            flag['publish'] = tmp[i + 1]
        elif tmp[i] in '出版年':
            flag['year'] = tmp[i + 1]
        elif tmp[i] in '页数':
            flag['page_num'] = tmp[i + 1]
        elif tmp[i] in '定价':
            flag['price'] = tmp[i + 1]
        elif tmp[i] in 'ISBN':
            flag['ISBN'] = tmp[i + 1]
        else:
            tmp_arr.append({tmp[i]: tmp[i + 1]})
    flag['other'] = tmp_arr


def requests_url(url):
    header['User-Agent'] = random.choice(user_agent)
    page = requests.get(url, headers=header)
    page.encoding = 'utf-8'
    return page.text


def init(flag):
    flag['type'] = '',
    flag['lable'] = '',
    flag['url'] = '',
    flag['name'] = '',
    flag['author'] = '',
    flag['publish'] = '',
    flag['year'] = '',
    flag['page_num'] = '',
    flag['price'] = '',
    flag['ISBN'] = '',
    flag['other'] = '',
    flag['img_url'] = '',
    flag['average'] = '',
    flag['content_validity'] = '',
    flag['author_validity'] = '',
    flag['indent'] = ''


start = time.time()

conn = pymysql.connect('localhost', 'root', '123', 'pc')
cursor = conn.cursor()

flag = {'type': '', 'label': '', 'url': url}
init(flag)
print(flag)
page_html = requests_url(url)
flag['name'] = re.findall(regx_get_book_info[0], page_html)[0]
information = re.sub(sub_regx[0], '', re.findall(regx_get_book_info[1], page_html)[0])
get_info_by_list(re.findall(regx_get_book_info[7], information), flag)
flag['img_url'] = re.findall(regx_get_book_info[2], page_html)[0]
flag['average'] = re.findall(regx_get_book_info[3], page_html)[0]
information = re.findall(regx_get_book_info[4], page_html)
if len(information) == 2:
    flag['content_validity'] = information[0]
    flag['author_validity'] = information[1]
elif len(information) == 1:
    if len(re.findall(regx_get_book_info[8], page_html)) == 1:
        flag['content_validity'] = information[0]
        flag['author_validity'] = ''
    else:
        flag['content_validity'] = ''
        flag['author_validity'] = information[0]
elif len(information) == 0:
    flag['content_validity'] = ''
    flag['author_validity'] = ''
try:
    flag['indent'] = re.findall(regx_get_book_info[5], page_html)[0]
except Exception:
    flag['indent'] = ''
print(flag)
sql = 'insert into douban_book_info_new values ("{}","{}","{}","{}","{}","{}","{}","{}","{}","{}","{}","{}","{}",\'{}\',\'{}\',\'{}\')'.format(
    flag['type'],
    flag['label'],
    flag['url'],
    flag['name'],
    flag['author'],
    flag['publish'],
    flag['year'],
    flag['page_num'],
    flag['price'],
    flag['ISBN'],
    flag['other'],
    flag['img_url'],
    flag['average'],
    flag['content_validity'].replace("'", '"'),
    flag['author_validity'].replace("'", '"'),
    flag['indent'].replace("'", '"')
)
print(sql)
cursor.execute(sql)
conn.commit()
